﻿''' <summary>
''' NWISDV: http://waterservices.usgs.gov/rest/WOF-IV-Service.html
''' See also: http://waterservices.usgs.gov/
''' </summary>
''' <remarks></remarks>
Public Class clsNwisDv
    Inherits clsWebProvider

    Private WithEvents NwisDVRes As New NwisDv.GetWSService

    Public Overrides ReadOnly Property Description As String
        Get
            Return "The USGS NWIS (U.S. Geological Survey National Water Information System) provides native access to USGS daily value data. For more information about NWIS, see the NWIS home page at http://waterdata.usgs.gov/nwis."
        End Get
    End Property

    Public Overrides ReadOnly Property URL As String
        Get
            Return "http://waterdata.usgs.gov/nwis"
        End Get
    End Property

    Protected Overridable Property SiteCode As String = "all"

    ''' <summary>
    ''' Get Stations
    ''' </summary>
    ''' <param name="NameFilter"></param>
    ''' <returns></returns>
    ''' <remarks>See http://waterservices.usgs.gov/rest/USGS-Site-Test-Tool.html for tool to build URL</remarks>
    Public Overrides Function GetStations(Optional NameFilter As String = "") As Boolean
        Cancelled = False
        Dim ds As DataSet = Nothing
        With AreaOfInterest
            Dim url As String = String.Format("http://interim.waterservices.usgs.gov/SiteFile/getSites?bbox={0},{1},{2},{3}&SiteType=sw,gw,sp,at,ot&SiteCode={4}&Format=WaterML", .Left, .Top, .Right, .Bottom, SiteCode)
            If Not DownloadNwisData(url, ds) OrElse ds Is Nothing Then Return False
        End With
        dictStations.Clear()
        Dim StationID As String = ""
        Dim StationName As String = ""
        Dim Lat As Double = Double.MaxValue
        Dim Lon As Double = Double.MaxValue
        Dim Network As String = ""

        For Each drSite As DataRow In ds.Tables("site").Rows
            For Each drSiteInfo As DataRow In drSite.GetChildRows("site_siteInfo")
                StationName = GetValue(drSiteInfo, "siteName")
                For Each drSiteCode As DataRow In drSiteInfo.GetChildRows("siteInfo_siteCode")
                    Network = GetValue(drSiteCode, "network")
                    StationID = GetValue(drSiteCode, "sitecode_text")
                Next
                For Each drGeoLocation As DataRow In drSiteInfo.GetChildRows("siteInfo_geolocation")
                    For Each drGeogLocation As DataRow In drGeoLocation.GetChildRows("geolocation_geoglocation")
                        Lat = GetValue(drGeogLocation, "latitude", "0")
                        Lon = GetValue(drGeogLocation, "longitude", "0")
                    Next
                Next
            Next
            StationID = Network & ":" & StationID
            If Not dictStations.ContainsKey(StationID) Then
                dictStations.Add(StationID, New clsStation(StationID, StationID, StationName, "", "USGS", "USGS", Lat, Lon, "", ""))
            End If
        Next

        'following returns too many stations; use alternate above so can segregate real-time stations
        'With AreaOfInterest
        '    If Not DownloadNwisData(String.Format("http://waterdata.usgs.gov/nwis/inventory?nw_longitude_va={0}&nw_latitude_va={1}&se_longitude_va={2}&se_latitude_va={3}&coordinate_format=decimal_degrees&sort_key=site_no&group_key=NONE&format=sitefile_output&sitefile_output_format=xml&column_name=agency_cd&column_name=site_no&column_name=station_nm&column_name=site_tp_cd&column_name=dec_lat_va&column_name=dec_long_va&column_name=state_cd&column_name=county_cd&column_name=alt_va&column_name=huc_cd&column_name=basin_cd&column_name=drain_area_va&list_of_search_criteria=lat_long_bounding_box", .Left, .Top, .Right, .Bottom), ds) Then Return False
        'End With
        'dictStations.Clear()
        'For Each r As DataRow In ds.Tables(0).Rows
        '    Dim agency As String = r.Item("agency_cd")
        '    Dim sta As String = "NWIS:" & r.Item("site_no")
        '    If Not dictStations.ContainsKey(sta) Then
        '        dictStations.Add(sta, New clsStation(sta,
        '                                            sta,
        '                                            r.Item("station_nm"),
        '                                            r.Item("site_tp_cd"),
        '                                            agency,
        '                                            agency,
        '                                            r.Item("dec_lat_va"),
        '                                            r.Item("dec_long_va"),
        '                                            r.Item("state_cd") & r.Item("county_cd"),
        '                                            r.Item("huc_cd")))
        '    End If
        'Next
        dictAgencies.Clear()
        dictAgencies.Add("USGS", New clsAgency("USGS", "U.S. Geological Survey"))
        Return True
    End Function

    Public Overrides Function GetParameters(Optional NameFilter As String = "") As Boolean
        Cancelled = False
        Static FileContents As String = ""
        If String.IsNullOrEmpty(FileContents) Then
            If Not DownloadNwisData("http://nwis.waterdata.usgs.gov/nwis/pmcodes/pmcodes?radio_pm_search=param_group&pm_group=Physical&format=rdb&show=parameter_nm&show=parameter_units", FileContents) Then Return False
        End If

        dictParameters.Clear()
        For Each s As String In FileContents.Split(vbLf)
            If Not (String.IsNullOrEmpty(s) OrElse s.StartsWith("#")) AndAlso IsNumeric(s.Substring(0, 5)) Then
                Dim ar() As String = s.Split(vbTab)
                Dim Parm As String = ar(1)
                If NameFilter = "" OrElse Parm.ToUpper.Contains(NameFilter.Trim.ToUpper) Then
                    dictParameters.Add("NWIS:" & ar(0), New clsParameter("", "NWIS:" & ar(0), ar(0), ar(1), ar(2), 0, Now, Now))
                End If
            End If
        Next
        Return True
    End Function

    Public Overrides Function GetResults() As Boolean
        Try
            LastException = Nothing

            Cancelled = False
            LastException = Nothing

            DataCount = 0
            lstData.Clear()

            Dim ds As DataSet = Nothing
            Dim cnt As Integer = 0
            Dim cntmax As Integer = SelectedStations.Count * SelectedParameters.Count

            'note: statistics codes are required: 00001=Max;00002=Min;00003=Mean;00006=Sum
            'for now, only do mean (for flow) and sum (for precip)

            For stat As Integer = 1 To 2
                Dim StatCode As String = Choose(stat, "00003", "00006")
                Dim SCode As String = Choose(stat, "AVG", "SUM")
                For Each sta As String In SelectedStations()
                    Dim lst As Generic.List(Of String) = SelectedParameters()
                    For Each parm As String In lst
                        Dim ParmNm As String = ""
                        If dictParameters.ContainsKey(parm) Then ParmNm = dictParameters(parm).ParmName
                        Dim sCount As String = ""
                        If SelectedStations.Count > 1 Then sCount = String.Format("; {0} values so far", DataCount)
                        If Not SetProgress(cnt, cntmax, String.Format("Getting results for {0} ({1}-{2}){3}...", Strings.Left(Strings.StrConv(sta, VbStrConv.ProperCase), 25), parm, SCode, sCount)) Then Return False
                        cnt += 1

                        Dim url As String = String.Format("http://waterservices.usgs.gov/NWISQuery/GetDV1?SiteNum={0}&StartDate={1}&EndDate={2}&ParameterCode={3}&StatisticCode={4}", sta.Replace("NWIS:", ""), StartDate.ToString("yyyy-MM-dd"), EndDate.ToString("yyyy-MM-dd"), parm.Replace("NWIS:", ""), StatCode)
                        If Not DownloadNwisData(url, ds) Then Return False 'Continue For

                        If ds Is Nothing OrElse ds.Tables.Count = 0 Then Continue For

                        If Not (ColumnExists(ds, "sitecode", "sitecode_Text") AndAlso ColumnExists(ds, "variablecode", "variablecode_Text")) Then Return False

                        Dim StationID As String = ""
                        Dim StationName As String = ""
                        Dim AgencyCode As String = ""
                        Dim Network As String = ""
                        Dim PCode As String = ""
                        Dim Units As String = ""
                        Dim CCode As String = ""
                        Dim QCode As String = ""
                        Dim RCode As String = ""
                        Dim DateTime As Date = Now
                        Dim Result As Double = Double.MaxValue

                        For Each drTimeSeries As DataRow In ds.Tables("timeseries").Rows
                            'note: am looping for all rows, but there should really only be a single child row
                            For Each drSourceInfo As DataRow In drTimeSeries.GetChildRows("timeseries_sourceInfo")
                                StationName = GetValue(drSourceInfo, "sitename")
                                For Each drSiteCode As DataRow In drSourceInfo.GetChildRows("sourceinfo_sitecode")
                                    AgencyCode = GetValue(drSiteCode, "agencycode")
                                    Network = GetValue(drSiteCode, "network")
                                    Network = "NWIS" 'override above because when discover stations, it uses NWIS instead of NWISDV
                                    StationID = Network & ":" & GetValue(drSiteCode, "sitecode_text")
                                Next
                            Next
                            For Each drVariable As DataRow In drTimeSeries.GetChildRows("timeseries_variable")
                                For Each drVariableCode As DataRow In drVariable.GetChildRows("variable_variablecode")
                                    PCode = Network & ":" & GetValue(drVariableCode, "variablecode_text")
                                Next
                                For Each drUnits As DataRow In drVariable.GetChildRows("variable_units")
                                    Units = GetValue(drUnits, "unitsAbbreviation")
                                Next
                            Next
                            For Each drValues As DataRow In drTimeSeries.GetChildRows("timeseries_values")
                                For Each drValue As DataRow In drValues.GetChildRows("values_value")
                                    QCode = Strings.Left(GetValue(drValue, "qualifiers"), 1)
                                    DateTime = GetValue(drValue, "datetime", Date.MinValue)
                                    Dim value As String = GetValue(drValue, "value_text")
                                    If value.Length > 0 AndAlso Not IsNumeric(value) Then 'assume has rcode before value
                                        RCode = GetValue(drValue, "censorCode")
                                        RCode = RCode.Replace("lt", "<").Replace("gt", ">").Replace("nc", "").Replace("nd", "<").Replace("pnq", "P")
                                        value = Val(value.Remove(0, 1))
                                    End If
                                    If IsNumeric(value) Then Result = Val(value) Else Result = Double.MaxValue
                                    If StationID <> "" AndAlso DateTime <> Date.MinValue AndAlso PCode <> "" AndAlso (Result <> Double.MaxValue OrElse RCode <> "") Then
                                        lstData.Add(New WRDB.Project.clsDataRecord(StationID, DateTime, PCode, Result, CCode, RCode, , , AgencyCode, , , , SCode, QCode))
                                        DataCount += 1
                                        If dictParameters.ContainsKey(PCode) Then
                                            dictParameters(PCode).Units = Units
                                        End If
                                    End If
                                Next
                            Next
                        Next
                    Next
                Next
            Next

            Return True
        Catch ex As Exception
            LastException = ex
            Return False
        End Try
    End Function

End Class

''' <summary>
''' USGS Instantaneous real-time data
''' </summary>
''' <remarks>
''' See:
''' http://waterservices.usgs.gov/rest/WOF-IV-Test-Tool.html
''' </remarks>
Public Class clsNwisIV
    Inherits clsNwisDv

    Protected Overrides Property SiteCode As String = "rtm"

    Public Overrides ReadOnly Property Description As String
        Get
            Return "The USGS NWIS (U.S. Geological Survey National Water Information System) provides native access to USGS real-time instantaneous value data within the last 120 days. For more information about NWIS, see the NWIS home page at http://waterdata.usgs.gov/nwis."
        End Get
    End Property

    Public Overrides ReadOnly Property URL As String
        Get
            Return "http://waterdata.usgs.gov/nwis"
        End Get
    End Property

    Public Overrides Function GetResults() As Boolean
        Try
            Cancelled = False
            LastException = Nothing

            DataCount = 0
            lstData.Clear()
            Dim ds As DataSet = Nothing
            Dim cnt As Integer = 0
            Dim cntmax As Integer = SelectedStations.Count * IIf(SelectedParameters.Count = dictParameters.Count, 1, SelectedParameters.Count)

            If Now.Subtract(MyBase.StartDate).Days > 120 Then
                LastException = New Exception("You can only retrieve data up to 120 days old.")
                Return False
            End If

            For Each sta As String In SelectedStations()
                Dim lst As Generic.List(Of String) = SelectedParameters()
                If lst.Count = dictParameters.Count Then 'all were selected
                    lst.Clear()
                    lst.Add("")
                End If
                For Each parm As String In lst
                    Dim ParmNm As String = "All parameters"
                    If dictParameters.count = 0 Then
                        ParmNm = parm
                    Else
                        If dictParameters.ContainsKey(parm) Then ParmNm = dictParameters(parm).ParmName
                    End If

                    Dim sCount As String = ""
                    If SelectedStations.Count > 1 Or lst.Count > 1 Then sCount = String.Format("; {0} values so far", DataCount)

                    If Not SetProgress(cnt, cntmax, String.Format("Getting results for {0} ({1}){2}...", sta, ParmNm, sCount)) Then Return False

                    Dim url As String = String.Format("http://waterservices.usgs.gov/nwis/iv?sites={0}&startDT={1}&endDT={2}{3}",
                                                      sta.Replace("NWIS:", ""),
                                                      StartDate.ToString("yyyy-MM-dd"),
                                                      EndDate.ToString("yyyy-MM-dd"),
                                                      IIf(parm = "", "", "&parameterCd=" & parm.Replace("NWIS:", "")))
                    If Not DownloadNwisData(url, ds) Then Return False
                    cnt += 1

                    'copied from bwGetResultsCuahsi

                    If ds.Tables.Count = 0 Then Continue For

                    If Not (ColumnExists(ds, "sitecode", "sitecode_Text") AndAlso ColumnExists(ds, "variablecode", "variablecode_Text")) Then Return False

                    Dim StationID As String = ""
                    Dim StationName As String = ""
                    Dim AgencyCode As String = ""
                    Dim Network As String = ""
                    Dim PCode As String = ""
                    Dim Units As String = ""
                    Dim CCode As String = ""
                    Dim QCode As String = ""
                    Dim RCode As String = ""
                    Dim DateTime As Date = Now
                    Dim Result As Double = Double.MaxValue

                    For Each drTimeSeries As DataRow In ds.Tables("timeseries").Rows
                        'note: am looping for all rows, but there should really only be a single child row
                        For Each drSourceInfo As DataRow In drTimeSeries.GetChildRows("timeseries_sourceInfo")
                            StationName = GetValue(drSourceInfo, "sitename")
                            For Each drSiteCode As DataRow In drSourceInfo.GetChildRows("sourceinfo_sitecode")
                                AgencyCode = GetValue(drSiteCode, "agencycode")
                                Network = GetValue(drSiteCode, "network")
                                StationID = Network & ":" & GetValue(drSiteCode, "sitecode_text")
                            Next
                        Next
                        For Each drVariable As DataRow In drTimeSeries.GetChildRows("timeseries_variable")
                            For Each drVariableCode As DataRow In drVariable.GetChildRows("variable_variablecode")
                                PCode = Network & ":" & GetValue(drVariableCode, "variablecode_text")
                            Next
                            'If ColumnExists(ds, "units", "unitsAbbreviation") Then Units = ds.Tables("units").Rows(0).Item("unitsAbbreviation")
                        Next
                        For Each drValues As DataRow In drTimeSeries.GetChildRows("timeseries_values")
                            For Each drValue As DataRow In drValues.GetChildRows("values_value")
                                QCode = GetValue(drValue, "qualifiers")
                                DateTime = GetValue(drValue, "datetime", Date.MinValue)
                                Dim value As String = GetValue(drValue, "value_text")
                                If value.Length > 0 AndAlso Not IsNumeric(value) Then 'assume has rcode before value
                                    RCode = GetValue(drValue, "censorCode")
                                    RCode = RCode.Replace("lt", "<").Replace("gt", ">").Replace("nc", "").Replace("nd", "<").Replace("pnq", "P")
                                    value = Val(value.Remove(0, 1))
                                End If
                                If IsNumeric(value) Then Result = Val(value) Else Result = Double.MaxValue
                                If StationID <> "" AndAlso DateTime <> Date.MinValue AndAlso PCode <> "" AndAlso (Result <> Double.MaxValue OrElse RCode <> "") Then
                                    lstData.Add(New WRDB.Project.clsDataRecord(StationID, DateTime, PCode, Result, CCode, RCode, , , , , , , , QCode))
                                    DataCount += 1
                                End If
                            Next
                        Next
                    Next
                    'Dim StationID As String = ""
                    'With ds.Tables("sitecode").Rows(0)
                    '    StationID = .Item("Network") & ":" & .Item("sitecode_Text")
                    'End With
                    'Dim StationName As String = ds.Tables("sourceInfo").Rows(0).Item("siteName")
                    'Dim PCode As String = ""
                    'With (ds.Tables("variablecode").Rows(0))
                    '    PCode = .Item("network") & ":" & .Item("variablecode_Text")
                    'End With
                    'With ds.Tables("variable").Rows(0)
                    '    Dim DataType As String = ""
                    '    If ColumnExists(ds, "variable", "dataType") Then DataType = TestNull(.Item("dataType"), "")
                    '    If Not String.IsNullOrEmpty(DataType) Then PCode &= "/DataType=" & .Item("dataType") 'Else Debug.Print("not")
                    '    Dim ParmName As String = ds.Tables("variable").Rows(0).Item("variableName")
                    'End With
                    'Dim Units As String = ""
                    'If ColumnExists(ds, "units", "unitsAbbreviation") Then Units = ds.Tables("units").Rows(0).Item("unitsAbbreviation")
                    'Dim DepthM As String = ""

                    'If StationID <> "" AndAlso PCode <> "" AndAlso
                    '   ColumnExists(ds, "value", "DateTime") AndAlso
                    '   ColumnExists(ds, "value", "Value_Text") Then
                    '    With ds.Tables("value")
                    '        For i As Integer = 0 To .Rows.Count - 1
                    '            Dim DateTime As Date = .Rows(i).Item("DateTime")
                    '            Dim Result As String = .Rows(i).Item("Value_Text")
                    '            Dim RCode As String = ""
                    '            If ColumnExists(ds, "value", "censorCode") Then RCode = TestNull(.Rows(i).Item("censorCode"), "")
                    '            If RCode <> "" Then RCode = RCode.Replace("lt", "<").Replace("gt", ">").Replace("nc", "").Replace("nd", "<").Replace("pnq", "P")
                    '            lstData.Add(New WRDB.Project.clsDataRecord(StationID, DateTime, PCode, Result, , RCode, , DepthM))
                    '            DataCount += 1
                    '        Next
                    '    End With
                    '    If dictParameters.ContainsKey(PCode) Then dictParameters(PCode).Units = Units
                    'End If
                Next
            Next
            Return True
        Catch ex As Exception
            LastException = ex
            Return False
        End Try
    End Function
End Class
